import os

# 文件合并
""" 
mergefiledir = os.getcwd()+'\\stopwords'
filenames = os.listdir(mergefiledir)  # stopwords文件夹下的文件，四个停用词表
file = open('stopwords.txt','w')

for filename in filenames:
    filepath = mergefiledir + '\\' + filename
    for line in open(filepath):
        file.writelines(line)
    file.write('\n')
 """
# 文件去重

lines = open('stopwords.txt','r')
newfile = open('stopword.txt','w')
new  = []
for line in lines.readlines():
    if line not in new:
        new.append(line)
        newfile.writelines(line)

# file.close()
newfile.close()